package com.eagle.roomcrawler.crawlerdata;

import com.eagle.roomcrawler.entity.Url;
import com.eagle.roomcrawler.resolver.DataResolver;
import com.eagle.roomcrawler.urlmanager.UrlManager;
import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.Connection.Method;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

/**
 * jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。
 * 它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据
 * 网页下载器
 */
@Component
public class HttpExecute {
    @Autowired
    DataResolver dataResolver;
    private static int TIMEOUT = 4000;
    public static void main(String[] args){
        try {
            Response response = execute("");
            int num = DataResolver.getPageNum(response.parse());
            System.out.println(num);
//            DataResolver.parsePageData(response.parse());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static String crawlerUrl() throws Exception {
        //从队列中取出并移除头元素的方法,  poll: 若队列为空，返回null。
        String url = UrlManager.configUrls.poll();
        Response response = execute(url);
        int num = DataResolver.getPageNum(response.parse());
        System.out.println("num============================="+num);
        if(num < 1){
            return "2";//不可用
        }
        for(int i = 1;i <= num;i++) {
            String zUrl = "";
            if(1 == i){
                zUrl = url+"/house";
            }else{
                zUrl = url+"/house/i3"+i;
            }
            System.out.println(zUrl);
            UrlManager.pushUrlToQueue(zUrl,"CRAWLER");
        }
        return "1";
    }

    public void crawlerThread(String rUrl) throws Exception {

        new Thread(new Runnable() {
            @Override
            public void run() {
//              DataResolver dataResolver = new DataResolver();
                String url = "";
                boolean flag = true;
                while(flag){
                    try {
                        if(UrlManager.crawlerUrls.size()<1){
                            flag = false;
                            System.out.println("数据爬取完成！");
                        }
                        url = UrlManager.crawlerUrls.poll();
                        if(url != null){
                            Response response = execute(url);
                            dataResolver.parsePageData(response.parse(),rUrl);
                        }
                        Thread.sleep(3000);
                    } catch (Exception e) {
                        e.printStackTrace();
                        UrlManager.pushUrlToQueue(url,"CRAWLER");
                    }
                }
            }
        }).start();
    }

    /**
     * 发送http请求
     * @param url
     * @return
     * @throws Exception
     */
    public static Response execute(String url) throws Exception{

        Connection connection = Jsoup.connect(url);
        connection.method(Method.GET).validateTLSCertificates(false).timeout(TIMEOUT)
                .ignoreContentType(true)
                .header("Host", "zu.sz.fang.com")
                .header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
        Response response=connection.execute();
        return response;

    }
}
